﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using HtmlAgilityPack;
using System.Text.RegularExpressions;
using System.Net;
using System.IO;
using System.IO.Compression;

namespace ConsoleApp
{
    /// <summary>
    /// 网络爬虫
    /// 2012-11-08（十八大召开的日子）
    /// ChengNing
    /// 
    /// 说明：使用HtmlAgilityPack包来辅助解析HTML
    /// </summary>
    class Spider
    {
        /// <summary>
        /// 爬取指定url的html内容
        /// 
        /// 核心内容
        /// request
        /// response
        /// response.GetResponseStream()
        /// sr = new StreamReader(stream, Encoding.GetEncoding(encode));
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public string Spid(string url)
        {
            string currentUrl = url;

            //A as B =====A is B？A：null
            HttpWebRequest request = WebRequest.Create(currentUrl) as HttpWebRequest;

            HttpWebResponse response = request.GetResponse() as HttpWebResponse;

            string encode = string.Empty;

            if (response.CharacterSet == "ISO-8859-1")
                encode = "gb2312";
            else
                encode = response.CharacterSet;


            Stream stream = response.GetResponseStream();
            //如果有被压缩
            if (response.ContentEncoding.ToLower() == "gzip")
            {
                stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress);
            }
            StreamReader sr = new StreamReader(stream, Encoding.GetEncoding(encode));
            string html = sr.ReadToEnd();
            sr.Close();
            return html;
        }


        /// <summary>
        /// 测试方法
        /// </summary>
        public static void Test()
        {
            Spider spider = new Spider();
            string url = "http://www.cnblogs.com/";
            string html = spider.Spid(url);

            HtmlDocument document = new HtmlDocument();

            document.LoadHtml(html);

            //提取title
            var title = document.DocumentNode.SelectSingleNode("//title").InnerText;

            //提取keywords
            //var keywords = document.DocumentNode.SelectSingleNode("//meta[@name='Keywords']").Attributes["content"].Value;

            Console.WriteLine(html);
        }
    }
}
